import json

import click
from xjlibrary.database_moudel.sqlmoudle import AbsDatabase
from xjlibrary.database_moudel.sqlmoudle.sqlbuilder import Sqlite3Builder
from xjlibrary.our_file_dir import BaseDir

curpath = BaseDir.get_file_dir_absolute(__file__)
# 配置文件路径
dbpath = BaseDir.get_new_path(curpath, "db.ini")
print(dbpath)


def conn(sec):
    sqllite3 = AbsDatabase.get_sql_factory().sqlite_factory()
    # 传入的是配置文件和sec选项
    sqlite3_moudle = Sqlite3Builder(dbpath, sec).build_all().get_moudle()
    sqllite3.link(sqlite3_moudle)
    # 返回一个连接
    return sqllite3


@click.command()
@click.option('--inputdir', default='', type=str, help='请输入需要转换的文件的目录')
@click.option('--outputdir', default='', type=str, help='请输入输出目录')
def file_line_add(inputdir, outputdir):
    sqlconn = conn("sqlite32")
    sql = '''insert or ignore into `bigfile`(`lngid`,`line`) values(?,?)'''
    for filePath in BaseDir.get_dir_all_files(inputdir):
        listsql = []
        print(filePath)
        for line in BaseDir.read_file_r_mode_yield(filePath):
            dicts = json.loads(line)
            doi = dicts["doi"]
            print(doi)
            listsql.append((doi, line))
            if len(listsql) > 10000:
                sqlconn.executemany(sql, listsql)
                sqlconn.my_commit()
                listsql.clear()
        sqlconn = conn("sqlite32")
        sqlconn.executemany(sql, listsql)
        sqlconn.my_commit()
        listsql.clear()


if __name__ == "__main__":
    file_line_add()
